#!/bin/bash -l
#PBS -o out
#PBS -e err
#PBS -N @[JBNME]
#PBS -A @[ACCNR]
#PBS -q @[QUEUE]
#PBS -l select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]:ompthreads=@[THRD]
#PBS -l walltime=00:@[WLCLK]:00

set -eux
echo -n " $( date +%s )," >  job_timestamp.txt

set +x
MACHINE_ID=derecho
source ./module-setup.sh
if [[ ! -d modulefiles ]]; then
  mkdir modulefiles
  mv *.lua modulefiles
fi
module use $PWD/modulefiles
module load modules.fv3
module list
set -x

echo "Model started:  " `date`

export MPI_TYPE_DEPTH=20
export OMP_STACKSIZE=512M
export OMP_NUM_THREADS=@[THRD]
export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
export ESMF_RUNTIME_PROFILE=ON
export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
export MPICH_COLL_OPT_OFF=1

# Avoid job errors because of filesystem synchronization delays
sync && sleep 1

# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run.
if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then
    echo "The job should abort now, with exit status 1." 1>&2
    echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2
    false
fi

mpiexec -n @[UFS_TASKS] -ppn @[PPN] --hostfile $PBS_NODEFILE ./fv3.exe

echo "Model ended:    " `date`
echo -n " $( date +%s )," >> job_timestamp.txt
